package com.dmcb.trade.business.crawlers.article;

import com.dmcb.common.business.constants.CommonConstant;
import com.dmcb.common.business.services.BaseService;
import com.dmcb.common.business.utils.*;
import com.dmcb.common.web.conversion.JsonResult;
import com.dmcb.common.business.utils.WebClientUtil;
import com.dmcb.trade.business.constants.CrawlConstant;
import com.dmcb.trade.business.constants.TradeConstant;
import com.dmcb.trade.business.crawlers.author.ToutiaoCrawler;
import com.dmcb.trade.business.entities.*;
import com.dmcb.trade.business.mappers.ArticleCrawlMapper;
import com.dmcb.trade.business.mappers.ArticleMonitorMapper;
import com.gargoylesoftware.htmlunit.html.HtmlPage;
import org.apache.commons.lang3.StringUtils;
import org.jsoup.Jsoup;
import org.jsoup.nodes.Document;
import org.jsoup.nodes.Element;
import org.jsoup.select.Elements;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.springframework.beans.factory.annotation.Autowired;

import java.io.UnsupportedEncodingException;
import java.net.URL;
import java.util.*;

/**
 * 文章爬取解析基础类
 * Created by long on 2016/3/12.
 */
public abstract class BaseParser extends BaseService {

    Logger logger = LoggerFactory.getLogger(BaseParser.class);

    @Autowired
    private ArticleCrawlMapper articleCrawlMapper;
    @Autowired
    private ArticleMonitorMapper articleMonitorMapper;
    @Autowired
    private WebClientUtil webClientUtil;

    /**
     * @return 平台名称
     */
    public String getPlatForm() {
        return platform();
    }

    /**
     * 爬取、解析文章信息
     *
     * @param authorId 作者ID
     * @param link     文章链接，如：http://mt.sohu.com/20160311/n440124935.shtml
     * @return 结果消息
     * @throws Exception
     */
    public JsonResult parse(int authorId, String link) throws Exception {
        Author author = articleCrawlMapper.queryAuthor(authorId);
        author = ToutiaoCrawler.getAuthor(authorId, author, articleCrawlMapper);
        Document doc = getDocument(link);
        if (doc == null || doc.text().length() < 50) {
            //如果解析失败，且文章链接为头条链接，则后台替换IP
            if (RegexUtil.startWith(link, CrawlConstant.TOUTIAO_URL)) {
                RedisUtils.changeIP(15);
            }
            return error(CommonConstant.ABNORMAL_MSG);
        }
        Article article = checkOrInit(doc, authorId, link, false);
        if (article == null) {
            return error(TradeConstant.MSG_ERROR_TITLE_NONE);
        }
        // 如果已经存在，直接返回
        if (article.getId() > 0) {
            return warn(article.getId(), TradeConstant.MSG_ERROR_TITLE_EXIST);
        }
        // 解析发布日期、内容、封面等
        parse(doc, article, false);
        // 设置默认值、保存
        return success(save(author, article, platform()));
    }

    /**
     * 爬取、解析微信文章信息，保存到盗载监测
     *
     * @param monitorWechat 微信号记录
     * @param link          文章链接，如：http://mt.sohu.com/20160311/n440124935.shtml
     * @return 结果消息
     * @throws Exception
     */
    public JsonResult parseWeChatMonitor(MonitorWechat monitorWechat, String link) throws Exception {
        Document doc = getDocument(link);
        if (doc == null || doc.text().length() < 50) {
            //如果解析失败，且文章链接为头条链接，则后台尝试替换IP
            if (RegexUtil.startWith(link, CrawlConstant.TOUTIAO_URL)) {
                RedisUtils.changeIP(15);
            }
            return error(CommonConstant.ABNORMAL_MSG);
        }
        // 校验文章
        Article article = checkOrInit(doc, monitorWechat.getId(), link, true);
        if (article == null) {
            return error(TradeConstant.MSG_ERROR_TITLE_NONE);
        }
        // 如果已经存在，直接返回
        if (article.getId() > 0) {
            return warn(article.getId(), TradeConstant.MSG_ERROR_TITLE_EXIST);
        }
        // 解析发布日期、内容、封面等
        parse(doc, article, true);
        // 保存监测文章
        articleMonitorMapper.insertMonitorArticle(article);
        return success(article.getId());
    }

    /**
     * 根据文章链接获取文章内容所有图片
     *
     * @param link 文章链接
     * @return 结果
     */
    public JsonResult getImageList(String link) {
        Document doc = getDocument(link);
        if (doc == null) {
            return error(CommonConstant.ABNORMAL_MSG);
        }
        String title = text(doc, titleSelect());
        if (StringUtils.isBlank(title)) {
            return error("获取标题失败");
        }
        Element element = select(doc, bodySelect());
        if (element == null) {
            return error(CommonConstant.ABNORMAL_MSG);
        }
        Elements elements = element.select("img");
        if (elements.size() == 0) {
            return warn(null, "获取图片列表失败");
        }
        List<String> strings = new ArrayList<>();
        String src;
        for (Element element1 : elements) {
            src = element1.absUrl(imageType());
            src = StringUtils.isBlank(src) ? element1.attr("data-src") : src;
            if (StringUtils.isBlank(src)) {
                continue;
            }
            strings.add(src);
        }
        Map<String, Object> map = new HashMap<>();
        map.put("title", title);
        map.put("images", strings);
        return success(map);
    }

    /**
     * 爬取、解析文章信息，保存到盗载监测
     *
     * @param userId 作者ID
     * @param link   文章链接，如：http://mt.sohu.com/20160311/n440124935.shtml
     * @return 结果消息
     * @throws Exception
     */
    public JsonResult parseMonitor(int userId, String link) throws Exception {
        Document doc = getDocument(link);
        if (doc == null) {
            return error(CommonConstant.ABNORMAL_MSG);
        }
        // 查询或新增用户关联监测号
        MonitorWechat monitorWechat = articleMonitorMapper.queryByUserId(userId);
        if (monitorWechat == null) {
            monitorWechat = new MonitorWechat();
            monitorWechat.setUserId(userId);
            articleMonitorMapper.insertWechat(monitorWechat);
        }
        // 校验文章
        Article article = checkOrInit(doc, monitorWechat.getId(), link, true);
        if (article == null) {
            return error(TradeConstant.MSG_ERROR_TITLE_NONE);
        }
        // 如果已经存在，直接返回
        if (article.getId() > 0) {
            return warn(article.getId(), TradeConstant.MSG_ERROR_TITLE_EXIST);
        }
        // 解析发布日期、内容、封面等
        parse(doc, article, true);
        // 保存监测文章
        articleMonitorMapper.insertMonitorArticle(article);
        return success(article.getId());
    }

    /**
     * 爬取、解析文章信息 结果返回文章标题、时间
     *
     * @param link 文章链接
     * @return 结果消息
     */
    public JsonResult parse(String link) {

        Document doc = getDocument(link);
        String title = text(doc, titleSelect());

        if (doc == null || doc.text().length() < 50) {
            //如果解析失败，且文章链接为头条链接，则后台替换IP
            if (RegexUtil.startWith(link, CrawlConstant.TOUTIAO_URL)) {
                RedisUtils.changeIP(15);
            }
            return error(CommonConstant.ABNORMAL_MSG);
        }
        title = titleProcess(doc, title);
        Element element = isDateLast() ? doc.select(dateSelect()).last() : doc.select(dateSelect()).first();
        Map<String, String> map = new HashMap<>();
        map.put("data", text(element));
        map.put("title", title);
        map.put("author", getAuthorName(doc));
        String body = getBody(doc);
        if (StringUtils.isNotBlank(body)) {
            body = HtmlUtil.getTextFromHtml(body);
        }
        map.put("content", body);
        return success(map);
    }


    /**
     * 解析获取Body
     *
     * @param link 链接
     * @return 内容
     */
    public String parseBody(String link) {
        Document doc = getDocument(link);
        if (doc == null) {
            return "";
        }
        return getBody(doc);
    }

    /**
     * 解析Body
     *
     * @param xml 页面XML
     * @return 内容
     */
    public String parseBodyFromXml(String xml) {
        if (StringUtils.isBlank(xml)) {
            return "";
        }
        Document doc = Jsoup.parse(xml);
        if (doc == null) {
            return "";
        }
        return getBody(doc);
    }

    public Document getDocument(String link) {
        if (!RegexUtil.startWithDomain(link, url())) {
            logger.error("链接前缀有误，link:" + link + "url：{}", url());
            return null;
        }
        try {
            HtmlPage htmlPage;
            if (isProxyEnabled()) {
                WebClientUtil.wc = webClientUtil.initWc(isJsEnabled(), false, false);
                htmlPage = webClientUtil.getPage(link);
            } else {
                htmlPage = isJsEnabled() ? webClientUtil.getPageJs(link, true) : webClientUtil.getPage(link);
            }
            // 文章解析
            if (htmlPage == null) {
                return Jsoup.parse(new URL(link), 6000);
            }
            return Jsoup.parse(htmlPage.asXml(), baseUri(link));
        } catch (Exception e) {
            logger.error("getDocument error", link, e);
            return null;
        }
    }

    private String baseUri(String link) {
        if (link.contains("?")) {
            link = link.substring(0, link.indexOf("?"));
        }
        return link.substring(0, link.lastIndexOf("/") + 1);
    }

    /**
     * 校验文章是否已存在
     *
     * @param doc      网页内容
     * @param authorId 作者ID
     * @return 文章
     */
    public Article checkOrInit(Document doc, int authorId, String link, boolean monitor) {
        String title = text(doc, titleSelect());
        title = titleProcess(doc, title);
        if (StringUtils.isBlank(title)) {
            logger.error("标题不存在, link={}", link);
            if (link.startsWith("http://www.toutiao.com/")) {
                logger.info(doc.toString());
            }
            return null;
        }
        // 校验该作者标题是否存在
        Article article = monitor ? articleMonitorMapper.queryByUserTitle(authorId, title) : articleCrawlMapper.queryByAuthorTitle(authorId, title);
        if (article != null) {
            return article;
        }
        return initArticle(authorId, title, link);
    }

    /**
     * 初始化文章
     *
     * @param authorId 作者ID
     * @param title    标题
     * @param link     链接
     * @return 文章
     */
    public Article initArticle(int authorId, String title, String link) {
        // 新文章
        Article article = new Article();
        article.setAuthorId(authorId);
        try {
            article.setTitle(EncodeUtil.filterOffUtf8Mb4(title));
        } catch (UnsupportedEncodingException e) {
            e.printStackTrace();
        }
        article.setLink(link);
        return article;
    }

    /**
     * 设置默认值、保存
     *
     * @param author  作者
     * @param article 文章
     * @return 文章ID
     */
    public int save(Author author, Article article, String platForm) {
        // 设置昵称
        try {
            String authorName;
            if (StringUtils.isBlank(article.getAuthorName())) {
                authorName = author.getAuthorName();
                if (StringUtils.isNotBlank(authorName) && !authorName.equals("匿名")) {
                    article.setAuthorName(authorName);
                }
            }
            // 设置价格
            article.setPrice(author.getPrice());
            article.setMaterialPrice(author.getMaterialPrice());
            article.setMaterialAllow(author.getMaterialAllow());
            // 设置类别
            article.setGenre(author.getGenre());
            article.setApproveStatus(Article.STATUS_WAITING); // 审批中
            article.setShelfStatus(Article.STATUS_OFF_SHELF); // 未上架
            // 设置平台名称
            article.setPlatform(platForm);
            article.setScore("A".equals(author.getRank()) ? 6 : 4);
            authorName = article.getAuthorName();
            if (StringUtils.isNotBlank(authorName)) {
                article.setAuthorName(EncodeUtil.filterOffUtf8Mb4(authorName));
            }
            try {
                articleCrawlMapper.insert(article);
            } catch (Exception e1) {
                article.setTitle(EncodeUtil.filterOffUtf8Mb4(article.getTitle()));
                article.setBody(EncodeUtil.filterOffUtf8Mb4(article.getBody()));
                article.setPlatform(EncodeUtil.filterOffUtf8Mb4(article.getPlatform()));
                articleCrawlMapper.insert(article);
            }
        } catch (UnsupportedEncodingException e) {
            e.printStackTrace();
        }

        return article.getId();
    }

    /**
     * 获取元素的文本内容
     *
     * @param select 选择器
     * @return 内容
     */
    protected String text(Document doc, String select) {
        return text(select(doc, select));
    }

    /**
     * 获取元素的文本内容
     *
     * @param element 元素
     * @return 内容
     */
    protected String text(Element element) {
        if (element == null) {
            return "";
        }
        String text = element.text();
        return StringUtils.isBlank(text) ? "" : text.trim();
    }

    /**
     * 解析网页文章，设置发布日期、正文、封面、作者
     *
     * @param doc     网页内容
     * @param article 文章
     */
    public void parse(Document doc, Article article, boolean monitor) {
        // 发布日期
        setPostDate(doc, article);
        // 正文
        setBody(doc, article, monitor);
        // 封面
        setCoverPath(doc, article);
        // 作者名称
        setAuthorName(doc, article);
    }

    /**
     * 获取内容
     */
    public String getBody(Document doc) {
        Element element = select(doc, bodySelect());
        if (element == null) {
            return "";
        }
        String body = element.html();
        if (StringUtils.isBlank(body)) {
            return "";
        }
        return body.trim();
    }

    /**
     * 保存、设置内容
     */
    protected void setBody(Document doc, Article article, boolean monitor) {
        String body = getBody(doc);
        if (StringUtils.isBlank(body)) {
            return;
        }
        body = HtmlUtil.getTextFromHtml(body);
        // 非监测，保存内容
        if (!monitor) {
            Body b = new Body(body);
            articleCrawlMapper.insertBody(b);
            // 设置内容ID、内容
            article.setBodyId(b.getId());
        }
        article.setBody(HtmlUtil.getTextFromHtml(body));
    }

    /**
     * 设置封面
     */
    protected void setCoverPath(Document doc, Article article) {
        // 获取图片
        String path = getCoverPath(doc);
        // 微信等校验、处理图片
        path = coverProcess(doc, path);
        if (StringUtils.isNotBlank(path)) {
            article.setCoverPath(path);
        }
    }

    /**
     * 获取封面图片
     *
     * @param doc 网页
     * @return 路径
     */
    private String getCoverPath(Document doc) {
        Element element = select(doc, coverSelect());
        if (element == null) {
            return null;
        }
        return src(element);
    }

    /**
     * 获取封面图片src属性值
     *
     * @param element 元素
     * @return 路径
     */
    protected String src(Element element) {
        String path = element.absUrl("src");
        if (StringUtils.isBlank(path)) {
            path = element.attr("data-src");
        }
        if (path.startsWith("data:")) {
            path = element.attr("data-src");
        }
        return path;
    }

    /**
     * 根据链接获取作者名称
     *
     * @param link
     * @return
     */
    public JsonResult getAuthorName(String link) {
        Document doc = getDocument(link);
        if (doc == null) {
            return error(CommonConstant.ABNORMAL_MSG);
        }
        String data;
        String time = "";
        String authorName = "";
        Map<String, String> resultMap = new HashMap<>();
        Map<String, Object> map;
        List<Map<String, Object>> mapList;
        String json = doc.toString();
        if (link.contains("toutiao.com")) {
            String key = "subInfo: ";
            if (json.contains(key)) {
                json = json.substring(json.indexOf(key));
                time = json.substring(json.indexOf("time: '") + 7);
                time = time.substring(0, time.indexOf("'"));
                json = json.substring(json.indexOf("source: '") + 9);
                authorName = json.substring(0, json.indexOf("',"));
            }
        } else if (link.contains("mparticle.uc.cn")) {
            String key = "wm_cid=";
            String id = link.substring(link.indexOf(key) + key.length());
            id = id.substring(0, id.indexOf("!!"));
            data = WebUtil.buildRequest("https://ff.dayu.com/contents/" + id + "?biz_id=1002&_fetch_author=1&_fetch_incrs=1", false);
            if (!data.contains("data")) {
                return error("获取失败，联系客服");
            }
            map = JsonUtil.getResult(data);
            map = (Map) map.get("data");
            time = (String) DateUtil.formatByStringAndT((String) map.get("created_at"), true);
            map = (Map) map.get("_author");
            authorName = (String) map.get("author_name");
        } else if (link.contains("qq.com")) {//https://new.qq.com/omn/20181002/20181002a0s2uy00
            String id;
            if (link.contains(".html")){
                id=link.substring(link.lastIndexOf("/") + 1);
                id=id.substring(0,id.lastIndexOf("."))+"00";
            }else {
                id = link.substring(link.lastIndexOf("/") + 1);
            }
            data = WebUtil.buildRequest("https://openapi.inews.qq.com/getQQNewsNormalContent?id=" + id + "&chlid=news_rss&refer=mobilewwwqqcom&otype=jsonp&ext_data=all&srcfrom=newsapp&callback=getNewsContentOnlyOutput", false);
            data = data.replace("getNewsContentOnlyOutput(", "");
            data = data.substring(0, data.lastIndexOf(")"));
            map = JsonUtil.getResult(data);
            if (map == null || map.size() == 0) {
                return null;
            }
            time = (String) map.get("pubtime");
            authorName = map.get("src") + "";
        } else {
            Article article = new Article();
            setAuthorName(doc, article);
            setPostDate(doc, article);
            time = DateUtil.formatMinutes2(article.getPostTime());
            authorName = article.getAuthorName();
        }


        resultMap.put("time", time);
        resultMap.put("authorName", authorName);
        return success(resultMap);
    }

    public String getAuthorName(Document doc) {
        Element element = select(doc, authorNameSelect());
        String authorName = text(element);
        if (StringUtils.isNotBlank(authorName)) {
            authorName = authorName.trim().replace("编辑", "").replace("原创", "").replace("作者", "").replace("：", "").replace(":", "").replace("来源", "");
            try {
                authorName = EncodeUtil.filterOffUtf8Mb4(authorName);
            } catch (UnsupportedEncodingException e) {
                e.printStackTrace();
            }
        }
        return authorName;
    }

    /**
     * 设置作者名称
     */
    protected void setAuthorName(Document doc, Article article) {
        String authorName = getAuthorName(doc);
        if (StringUtils.isNotBlank(authorName)) {
            article.setAuthorName(authorName);
        }
    }

    /**
     * 设置发布日期
     *
     * @param article 文章
     */
    public void setPostDate(Article article) {
        if (article == null) {
            return;
        }
        setPostDate(getDocument(article.getLink()), article);
    }

    /**
     * 设置发布日期
     *
     * @param doc     网页
     * @param article 文章
     */
    protected void setPostDate(Document doc, Article article) {
        if (doc == null) {
            return;
        }
        try {
            Element element = isDateLast() ? doc.select(dateSelect()).last() : doc.select(dateSelect()).first();
            if (element != null) {
                Date date = DateUtil.parseDate(text(element));
                if (date == null) {
                    return;
                }
                //如果解析后的时间大于当前时间，则，减去一年
                if (date.after(new Date())) {
                    date = DateUtil.getLastDay(DateUtil.format(date), -1, 0, 0);
                }
                if (date != null) {
                    article.setPostTime(date);
                }
            }
        } catch (Exception e) {
            e.printStackTrace();
        }
    }

    /**
     * 选择元素
     *
     * @param doc    网页
     * @param select 选择器
     * @return 元素
     */
    private Element select(Document doc, String select) {
        if (StringUtils.isBlank(select)) {
            return null;
        }
        return doc.select(select).first();
    }

    /**
     * 判断是否是封面
     *
     * @param imgPath 图片路径
     * @return 是、否
     */
    protected boolean isCover(String imgPath) {
        int[] widthHeight = ImageUtil.getWidthHeight(imgPath);
        return widthHeight[0] >= 200 && widthHeight[1] >= 120;
    }

    /**
     * 网站链接
     *
     * @return 链接
     */
    protected abstract String url();

    /**
     * 获取标题选择器
     *
     * @return 标题选择器
     */
    protected abstract String titleSelect();

    /**
     * @return 获取标题
     */
    public String getTitle(String link) {
        Document doc = getDocument(link);
        if (doc == null) {
            return "";
        }
        if (doc.text().contains(" 出错了！文章没有找到哦~") || doc.text().contains("网页跑丢了")){//一点资讯/网易
            return " 出错了！文章没有找到哦~";
        }
        return getTitle(doc);
    }

    public String getTitle(Document doc) {
        Element element = select(doc, titleSelect());
        if (element == null) {
            return "";
        }
        String body = element.html();
        if (StringUtils.isBlank(body)) {
            return "";
        }
        return body.trim();
    }

    /**
     * 获取日期选择器
     *
     * @return 发布日期选择器
     */
    protected abstract String dateSelect();

    /**
     * @return 获取日期
     */
    public String getDateStr(String link) {
        Document doc = getDocument(link);
        if (doc == null) {
            return "";
        }
        return getDateStr(doc);
    }

    public String getDateStr(Document doc) {
        Element element = select(doc, dateSelect());
        if (element == null) {
            return "";
        }
        String body = element.html();
        if (StringUtils.isBlank(body)) {
            return "";
        }
        return body.trim();
    }

    /**
     * 日期是否在末尾元素
     *
     * @return 默认否
     */
    protected boolean isDateLast() {
        return false;
    }

    /**
     * 获取正文内容选择器
     *
     * @return 正文内容选择器
     */
    protected abstract String bodySelect();

    /**
     * 获取封面选择器
     *
     * @return 封面选择器
     */
    protected String coverSelect() {
        return null;
    }

    /**
     * 获取封面选择器 多个
     *
     * @return 封面选择器
     */
    protected String coverSelectMulti() {
        return null;
    }

    /**
     * 封面图片路径校验、替换等处理
     *
     * @param path 原路径
     * @return 处理后路径
     */
    protected String coverProcess(Document doc, String path) {
        if (StringUtils.isBlank(coverSelectMulti())) {
            return path;
        }
        if (isCover(path)) {
            return path;
        }
        Elements elements = doc.select(coverSelectMulti());
        if (elements == null || elements.size() == 0) {
            return path;
        }
        for (Element element : elements) {
            String imgPath = src(element);
            if (isCover(imgPath)) {
                return imgPath;
            }
        }
        return path;
    }

    /**
     * 获取作者选择器
     *
     * @return 作者选择器
     */
    protected String authorNameSelect() {
        return null;
    }

    protected boolean isJsEnabled() {
        return false;
    }

    protected boolean isProxyEnabled() {
        return false;
    }

    protected String titleProcess(Document doc, String title) {
        return title;
    }

    protected abstract String platform();

    protected String imageType() {
        return "src";
    }
}
